# -*- coding: utf-8 -*-
# @Time    : 2019/11/26 10:04
# @Author  : ZSQ
# @Email   : zsq199170918@163.com
# @FileName: shenzhen.py
# @Software: PyCharm
import scrapy
from scrapy_redis.spiders import RedisSpider

import re
import logging
import urllib.parse
from datetime import datetime
from copy import copy, deepcopy
from FDC_spider.items import FdcEstateGuidItem, FdcBuildingItem, FdcRoomItem

logger = logging.getLogger(__name__)


# class ShenzhenSpider(scrapy.Spider):
class ShenzhenSpider(RedisSpider):
    name = 'shenzhen'
    allowed_domains = ['zjj.sz.gov.cn']
    start_urls = ['http://zjj.sz.gov.cn/ris/bol/szfdc/index.aspx']
    redis_key = 'shenzhen'
    page_size = 20

    def parse(self, response):
        """
        重构请求，减少请求数，加快爬取效率（默认10/页，改为20/页)
        :param response:
        :return:
        """
        # 获取页数
        page_num = 1
        total_num, total_page_num = self._get_total_page(response)
        page_dict = dict(page_num=page_num, total_num=total_num)
        # 获取当前页参数
        VIEWSTATE, VIEWSTATEGENERATOR, EVENTVALIDATION = self._get_params(response)
        # 构造POST请求参数
        data = {
            'scriptManager2': 'updatepanel2|AspNetPager1',
            '__EVENTTARGET': 'AspNetPager1',
            '__EVENTARGUMENT': str(page_num),
            '__LASTFOCUS': '',
            '__VIEWSTATE': VIEWSTATE,
            '__VIEWSTATEGENERATOR': VIEWSTATEGENERATOR,
            '__VIEWSTATEENCRYPTED': '',
            '__EVENTVALIDATION': EVENTVALIDATION,
            'tep_name': '',
            'organ_name': '',
            'site_address': '',
            'ddlPageCount': str(self.page_size),
        }
        # 获取预售许可证列表
        yield scrapy.FormRequest(
            self.start_urls[0],
            formdata=data,
            callback=self.parse_permit_li,
            meta=dict(page_dict=deepcopy(page_dict))
        )

    def parse_permit_li(self, response):
        """
        获取预售许可证列表
        :param response:
        :return:
        """
        # 获取页数
        page_dict = copy(response.meta['page_dict'])
        page_num = page_dict.get('page_num')
        total_num, total_page_num = self._get_total_page(response)
        print('>>>>>>>>第{}/{}页<<<<<<<<'.format(page_num, total_page_num))
        if total_num != page_dict['total_num']:
            update_num = total_num - page_dict['total_num']
            logger.warning('{}  许可证列表页更新，更新量为{}条'.format(self.get_current_time(), update_num))
        # 获取许可证分组
        th_li = response.xpath("//th[contains(text(),'序号')]/../following-sibling::tr[position()<21]")
        if th_li:
            for th in th_li:
                item_eg = FdcEstateGuidItem()
                item_eg['preSalePermit'] = th.xpath("./td[2]/a/text()").extract_first()
                item_eg['preSalePermit'] = self.clean_s(item_eg['preSalePermit'])
                item_eg['projectName'] = th.xpath("./td[3]/a/text()").extract_first()
                item_eg['projectName'] = self.clean_s(item_eg['projectName'])
                item_eg['projectUrl'] = th.xpath("./td[3]/a/@href").extract_first()
                item_eg['projectId'] = self._get_project_id(item_eg['projectUrl'])
                item_eg['developerName'] = th.xpath("./td[4]/text()").extract_first()
                item_eg['developerName'] = self.clean_s(item_eg['developerName'])
                item_eg['districtName'] = th.xpath("./td[5]/text()").extract_first()
                item_eg['districtName'] = self.format_district(self.clean_s(item_eg['districtName']))
                item_eg['certDate'] = th.xpath("./td[6]/text()").extract_first()
                item_eg['certDate'] = self.clean_s(item_eg['certDate'])
                # 获取项目详情
                yield response.follow(
                    item_eg['projectUrl'],
                    callback=self.parse_project,
                    meta=dict(item_eg=deepcopy(item_eg))
                )
        else:
            logger.error('第{}页 许可证列表获取为空'.format(page_num))

        # 翻页
        if int(page_num) < int(total_page_num):
            page_dict['page_num'] += 1
            # 获取当前页参数
            VIEWSTATE, VIEWSTATEGENERATOR, EVENTVALIDATION = self._get_params(response)
            # 构造POST请求参数
            data = {
                'scriptManager2': 'updatepanel2|AspNetPager1',
                '__EVENTTARGET': 'AspNetPager1',
                '__EVENTARGUMENT': str(page_dict['page_num']),
                '__LASTFOCUS': '',
                '__VIEWSTATE': VIEWSTATE,
                '__VIEWSTATEGENERATOR': VIEWSTATEGENERATOR,
                '__VIEWSTATEENCRYPTED': '',
                '__EVENTVALIDATION': EVENTVALIDATION,
                'tep_name': '',
                'organ_name': '',
                'site_address': '',
                'ddlPageCount': str(self.page_size),
            }
            yield scrapy.FormRequest(
                self.start_urls[0],
                formdata=data,
                callback=self.parse_permit_li,
                meta=dict(page_dict=deepcopy(page_dict))
            )

    def parse_project(self, response):
        """
        获取项目详情
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        item_eg['projectUrl'] = response.request.url
        item_eg['landPlotsNum'] = response.xpath(
            "//td/div[contains(text(),'宗地号')]/../following-sibling::td[1]/text()").extract_first()
        item_eg['landPlotsNum'] = self.clean_s(item_eg['landPlotsNum'])
        item_eg['projectAddress'] = response.xpath(
            "//td/div[contains(text(),'宗地位置')]/../following-sibling::td[1]/text()").extract_first()
        item_eg['projectAddress'] = self.clean_s(item_eg['projectAddress'])
        item_eg['surrenderDate'] = response.xpath(
            "//td/div[contains(text(),' 受让日期')]/../following-sibling::td[1]/text()").extract_first()
        item_eg['surrenderDate'] = self.clean_s(item_eg['surrenderDate'])
        item_eg['CA'] = response.xpath(
            "//td/div[contains(text(),'批准机关')]/../following-sibling::td[1]/div/text()").extract_first()
        item_eg['CA'] = self.clean_s(item_eg['CA'])
        item_eg['contractNumber'] = response.xpath(
            "//td/div[contains(text(),'合同文号')]/../following-sibling::td[1]/text()").extract_first()
        item_eg['contractNumber'] = self.clean_s(item_eg['contractNumber'])
        item_eg['landUsageTerm'] = response.xpath(
            "//td/div[contains(text(),'使用年限')]/../following-sibling::td[1]/div/text()").extract_first()
        item_eg['landUsageTerm'] = self.clean_s(item_eg['landUsageTerm'])
        item_eg['roomUse'] = response.xpath(
            "//td/div[contains(text(),'房屋用途')]/../following-sibling::td[1]/text()").extract_first()
        item_eg['roomUse'] = self.clean_s(item_eg['roomUse'])
        item_eg['landUse'] = response.xpath(
            "//td/div[contains(text(),'土地用途')]/../following-sibling::td[1]/text()").extract_first()
        item_eg['landUse'] = self.clean_s(item_eg['landUse'])
        item_eg['estateTotalArea'] = response.xpath(
            "//td/div[contains(text(),'总建筑面积')]/../following-sibling::td[1]/text()").extract_first()
        item_eg['estateTotalArea'] = self.clean_s(item_eg['estateTotalArea'])
        item_eg['preSaleRoomNo'] = response.xpath(
            "//td/div[contains(text(),'预售总套数')]/../following-sibling::td[1]/text()").extract_first()
        item_eg['preSaleRoomNo'] = self.clean_s(item_eg['preSaleRoomNo'])
        item_eg['preSaleArea'] = response.xpath(
            "//td/div[contains(text(),'预售总面积')]/../following-sibling::td[1]/text()").extract_first()
        item_eg['preSaleArea'] = self.clean_s(item_eg['preSaleArea'])
        yield item_eg

        # 获取楼栋分组
        tr_li = response.xpath("//th[contains(text(),'项目名称')]/../following-sibling::tr")
        if tr_li:
            for tr in tr_li:
                item_bd = FdcBuildingItem()
                item_bd['projectName'] = item_eg['projectName']
                item_bd['districtName'] = item_eg['districtName']
                item_bd['projectId'] = item_eg['projectId']
                item_bd['preSalePermit'] = item_eg['preSalePermit']
                item_bd['certDate'] = item_eg['certDate']
                item_bd['blockName'] = tr.xpath("./td[2]/text()").extract_first()
                item_bd['blockName'] = self.clean_s(item_bd['blockName'])
                item_bd['planningPermit'] = tr.xpath("./td[3]/text()").extract_first()
                item_bd['planningPermit'] = self.clean_s(item_bd['planningPermit'])
                item_bd['constructionPermit'] = tr.xpath("./td[4]/text()").extract_first()
                item_bd['constructionPermit'] = self.clean_s(item_bd['constructionPermit'])
                item_bd['buildingUrl'] = tr.xpath("./td[5]/a/@href").extract_first()
                item_bd['buildingUrl'] = urllib.parse.urljoin(response.request.url, item_bd['buildingUrl'])
                item_bd['buildingId'] = self._get_building_id(item_bd['buildingUrl'])
                yield item_bd
                # 获取楼栋座号
                yield scrapy.Request(
                    item_bd['buildingUrl'],
                    callback=self.parse_seat_li,
                    meta=dict(item_bd=deepcopy(item_bd)),
                )
        else:
            logger.warning(
                '{}  {}-{} 楼栋列表获取为空'.format(item_eg['projectUrl'], item_eg['districtName'], item_eg['projectName']))

    def parse_seat_li(self, response):
        """
        获取座号列表
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        # 获取座号分组
        a_li = response.xpath("//div[@id='divShowBranch']//a")
        if a_li:
            for a in a_li:
                seat_url = a.xpath("./@href").extract_first()
                seat_name = self._get_seat_name(seat_url)
                # 获取房号列表
                yield response.follow(
                    seat_url,
                    callback=self.parse_room_li,
                    meta=dict(item_bd=deepcopy(item_bd), seat_name=deepcopy(seat_name)),
                )
        else:
            logger.error(
                '{}  {}-{}-{}-{} 座号列表获取为空'.format(response.request.url, item_bd['districtName'], item_bd['projectName'],
                                                  item_bd['preSalePermit'], item_bd['blockName'], ))

    def parse_room_li(self, response):
        """
        获取房号列表
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        seat_name = copy(response.meta['seat_name'])
        # 获取房号分组
        td_li = response.xpath("//table//tr//td/div[contains(text(),'房号')]/..")
        if td_li:
            for td in td_li:
                item_rm = FdcRoomItem()
                item_rm['projectName'] = item_bd['projectName']
                item_rm['projectId'] = item_bd['projectId']
                item_rm['districtName'] = item_bd['districtName']
                item_rm['preSalePermit'] = item_bd['preSalePermit']
                item_rm['blockName'] = item_bd['blockName']
                item_rm['buildingId'] = item_bd['buildingId']
                item_rm['seatName'] = seat_name
                item_rm['roomUrl'] = td.xpath("./div/a/@href").extract_first()
                item_rm['saleStatus'] = td.xpath("./div/a/text()").extract_first()
                # 获取房号详情
                if item_rm['roomUrl']:
                    yield response.follow(
                        item_rm['roomUrl'],
                        callback=self.parse_room,
                        meta=dict(item_rm=deepcopy(item_rm)),
                    )
                else:
                    item_rm['roomNo'] = td.xpath("./div[1]/text()").extract_first()
                    item_rm['roomNo'] = self._get_room_no(item_rm['roomNo'])
                    item_rm['roomFloor'] = self.get_floor_num(self._get_floor_str(td))
                    item_rm['roomLocation'] = None
                    item_rm['unitPrice'] = None
                    item_rm['roomUse'] = None
                    item_rm['forecastBuildArea'] = None
                    item_rm['forecastInnerArea'] = None
                    item_rm['forecastSharedArea'] = None
                    yield item_rm
        else:
            logger.error(
                '{}  {}-{}-{}-{}-{}  房号列表获取为空'.format(response.request.url, item_bd['districtName'],
                                                      item_bd['projectName'],
                                                      item_bd['preSalePermit'], item_bd['blockName'],
                                                      seat_name if seat_name else '未命名', ))

    def parse_room(self, response):
        """
        获取房号详情
        :param response:
        :return:
        """
        item_rm = copy(response.meta['item_rm'])
        item_rm['roomUrl'] = response.request.url
        item_rm['roomLocation'] = response.xpath(
            "//td[contains(text(),'项目楼栋情况')]/following-sibling::td[1]/text()").extract_first()
        item_rm['roomLocation'] = self.clean_s(item_rm['roomLocation'])
        item_rm['unitPrice'] = response.xpath(
            "//td[contains(text(),'拟售价格')]/following-sibling::td[1]/text()").extract_first()
        item_rm['unitPrice'] = self.get_digit_str(item_rm['unitPrice'])
        item_rm['roomFloor'] = response.xpath(
            "//td[contains(text(),'楼层')]/following-sibling::td[1]/text()").extract_first()
        item_rm['roomFloor'] = self.get_floor_num(item_rm['roomFloor'])
        item_rm['roomNo'] = response.xpath(
            "//td[contains(text(),'房号')]/following-sibling::td[1]/text()").extract_first()
        item_rm['roomNo'] = self.clean_s(item_rm['roomNo'])
        item_rm['roomUse'] = response.xpath(
            "//td[contains(text(),'用途')]/following-sibling::td[1]/text()").extract_first()
        item_rm['roomUse'] = self.clean_s(item_rm['roomUse'])
        item_rm['forecastBuildArea'] = response.xpath(
            "//td[contains(text(),'建筑面积')]/following-sibling::td[1]/text()").extract_first()
        item_rm['forecastBuildArea'] = self.get_digit_str(item_rm['forecastBuildArea'])
        item_rm['forecastInnerArea'] = response.xpath(
            "//td[contains(text(),'户内面积')]/following-sibling::td[1]/text()").extract_first()
        item_rm['forecastInnerArea'] = self.get_digit_str(item_rm['forecastInnerArea'])
        item_rm['forecastSharedArea'] = response.xpath(
            "//td[contains(text(),'分摊面积')]/following-sibling::td[1]/text()").extract_first()
        item_rm['forecastSharedArea'] = self.get_digit_str(item_rm['forecastSharedArea'])
        yield item_rm

    @staticmethod
    def get_digit_str(_str):
        regex_1 = re.compile(r'\d+')
        regex_2 = re.compile(r'\d+.*\d+')
        if _str:
            ret_1 = regex_1.findall(_str)
            ret_2 = regex_2.findall(_str)
            if len(ret_1) > 1:
                return ret_2[0]
            elif len(ret_1) == 1:
                return ret_1[0]

    @staticmethod
    def get_current_time():
        return datetime.strftime(datetime.now(), '%Y-%m-%d %H:%M:%S')

    @staticmethod
    def _get_params(response):
        VIEWSTATE = response.xpath("//input[@id='__VIEWSTATE']/@value").extract_first()
        VIEWSTATEGENERATOR = response.xpath("//input[@id='__VIEWSTATEGENERATOR']/@value").extract_first()
        EVENTVALIDATION = response.xpath("//input[@id='__EVENTVALIDATION']/@value").extract_first()
        return VIEWSTATE, VIEWSTATEGENERATOR, EVENTVALIDATION

    def _get_total_page(self, response):
        total_num_str = response.xpath("//div[@id='SelectPageSizeDiv']/span/text()").extract_first()
        total_num = int(self.get_digit_str(total_num_str))
        total_page_num = total_num // self.page_size if not total_num % self.page_size else total_num // self.page_size + 1
        return total_num, total_page_num

    @staticmethod
    def _get_project_id(_str):
        regex = re.compile(r'id=(.*)')
        if _str:
            ret = regex.findall(_str)
            return ret[0] if ret else None

    @staticmethod
    def clean_s(_str):
        return _str.strip() if _str else None

    @staticmethod
    def format_district(_str):
        if _str and not _str.endswith('区'):
            _str += '区'
        return _str

    @staticmethod
    def _get_building_id(_str):
        regex = re.compile(r'id=(.*)&')
        if _str:
            ret = regex.findall(_str)
            return ret[0] if ret else None

    @staticmethod
    def _get_seat_name(_str):
        regex = re.compile(r'Branch=(.*)&isBlock')
        if _str:
            ret = regex.findall(_str)
            ret = urllib.parse.unquote(ret[0]) if ret else None
            return ret if ret != '未命名' else None

    @staticmethod
    def _get_floor_str(_td):
        floor_str = _td.xpath("./preceding-sibling::td[@rowspan]/div/text()").extract_first()
        if not floor_str:
            for i in range(1, 101):
                floor_str = _td.xpath(
                    "./../preceding-sibling::tr[{}]/td[@rowspan]/div/text()".format(i)).extract_first()
                if floor_str:
                    return floor_str
        return floor_str

    @staticmethod
    def get_floor_num(_str):
        regex_1 = re.compile(r'\d+-\d+')
        regex_2 = re.compile(r'-\d+|\d+')
        if _str:
            ret_1 = regex_1.findall(_str)
            if ret_1:
                return ret_1[0]
            ret_2 = regex_2.findall(_str)
            ret = ret_2[0] if ret_2 else _str
            if ret and '地下' in _str:
                return '-' + ret
            return ret

    @staticmethod
    def _get_room_no(_str):
        regex = re.compile(r'房号：(.*)')
        if _str:
            ret = regex.findall(_str)
            return ret[0] if ret else None
